#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# @Time: 2019/6/2  11:17
# @Author: 余浪人
# @email: yulangren520@gmail.com
from apps.lib.public import random_click
from apps.spiders.pipeline.item_img import save_img
from apps.spiders.crawler import download_get, strReplace, distinct_detail_write, conversion_code, distinct_url_verify
import datetime, re
from lxml import etree

from apps.spiders.pipeline.item import save_data


def xpath_analysis_detail(url, q_obj):
    response = download_get(url, q_obj.agreement_cls)
    if response and url:
        replace_con = q_obj.replace_content
        html = etree.HTML(conversion_code(response))
        try:
            title = re.sub(r"\s+", "", ''.join(html.xpath(f'{q_obj.title_rule}')))  # 去空白后的标题
            for rep in replace_con.splitlines():
                s, e = strReplace(rep)
                title = re.sub(s, e, title)
        except:
            title = ''
        try:
            content = '<br/>'.join(html.xpath(f'{q_obj.content_rule}'))  # 内容
            for rep in replace_con.splitlines():
                s, e = strReplace(rep)
                content = re.sub(s, e, content)
        except:
            content = ''
        try:
            author = ''.join(html.xpath(f'{q_obj.author_rule}'))  # 作者
        except:
            author = ''
        try:
            source = ''.join(html.xpath(f'{q_obj.source_rule}'))  # 来源
        except:
            source = ''
        try:
            keyword = ','.join(html.xpath(f'{q_obj.keys_rule}'))  # 关键词
        except:
            keyword = ''
        try:
            clicks = int(html.xpath(f'{q_obj.clicks_rule}')[0])  # 点击数
        except:
            clicks = 0
        try:
            description = ''.join(html.xpath(f'{q_obj.description_rule}'))  # 描述
            for rep in replace_con.splitlines():
                s, e = strReplace(rep)
                description = re.sub(s, e, description)
        except:
            description = content[:150]
        if title:
            save_data(title=title, superior=q_obj.class_lD, content=content, clicks=clicks, author=author,
                      keyword=keyword, description=description, source=source)


def re_analysis_detail(url, q_obj):
    response = download_get(url=url, agr_id=q_obj.agreement_cls)
    if response:
        response = conversion_code(response)
        replace_con = q_obj.replace_content
        try:
            s, e = strReplace(q_obj.title_rule)
            title = re.search(f'{s}(.*){e}', response, flags=re.S).group(1).lower()  # 标题
            title = re.sub(r"\s+", "", title)  # 去除空白字符
            for rep in replace_con:
                s, e = strReplace(rep)
                title = re.sub(s, e, title)
        except:
            title = ''
        title = re.sub(r"</?(.+?)?>", '', title)
        try:
            if q_obj.content_rule:
                s, e = strReplace(q_obj.content_rule)
                content = re.search(f'{s}(.*){e}', response, flags=re.S).group(1).lower()  # 内容
            else:
                content = ''
        except:
            content = ''
        try:
            if q_obj.keys_rule:
                s, e = strReplace(q_obj.keys_rule)
                keyword = re.search(f'{s}(.*){e}', response).group(1)  # 关键词
            else:
                keyword = ''
        except:
            keyword = ''
        try:
            if q_obj.author_rule:
                s, e = strReplace(q_obj.author_rule)
                author = re.search(f'{s}(.*){e}', response).group(1)  # 作者
            else:
                author = ''
        except:
            author = ''
        try:
            if q_obj.clicks_rule:
                s, e = strReplace(q_obj.clicks_rule)
                clicks = re.search(f'{s}(.*){e}', response).group(1)  # 点击
            else:
                clicks = random_click()
        except:
            clicks = random_click()
        try:
            if q_obj.description_rule:
                s, e = strReplace(q_obj.description_rule)
                description = re.search(f'{s}(.*){e}', response).group(1).lower()  # 描述
        except:
            description = ''
        try:
            if q_obj.source_rule:
                s, e = strReplace(q_obj.source_rule)
                source = re.search(f'{s}(.*){e}', response).group(1)  # 来源
            else:
                source = ''
        except:
            source = ''
        if q_obj.filterAll:
            content = re.sub(r"</?(.+?)?>", '', content)
        else:
            if q_obj.filterFrame:
                content = re.sub(r"</?iframe(.+?)?>", '', content, flags=re.S)
            if q_obj.filterObject:
                content = re.sub(r"</?object(.+?)?>", '', content, flags=re.S)
            if q_obj.filterScript:
                content = re.sub(r"<script(.+?)>.*</script>", '', content, flags=re.S)
            if q_obj.filterDiv:
                content = re.sub(r"</?div(.+?)?>", '', content, flags=re.S)
            if q_obj.filterStyle:
                content = re.sub(r"</?style(.+?)?>", '', content, flags=re.S)
            if q_obj.filterTable:
                content = re.sub(r"</?table(.+?)?>", '', content)
            if q_obj.filterSpan:
                content = re.sub(r"</?span(.+?)?>", '', content)
            if q_obj.filterImg:
                content = re.sub(r"<img(.*?)/?>", '', content, flags=re.S)
            if q_obj.filterFont:
                content = re.sub(r"</?font(.+?)?>", '', content)
            if q_obj.filterA:
                content = re.sub(r"</?a(.+?)?>", '', content)
            if q_obj.filterP:
                content = re.sub(r"</?p(.+?)?>", '', content)
            content = re.sub(r'<!--(.+?)?-->', '', content)
            content = re.sub(r'\t', '', content)
            content = re.sub(r'\n', '', content)
        if q_obj.img_is_save and q_obj.img_save_path:
            content = save_img(content, q_obj.agreement_cls, q_obj.img_save_path,
                               goal_url=q_obj.goal_url, ) if q_obj.img_relative else save_img(content,
                                                                                              q_obj.agreement_cls,
                                                                                              q_obj.img_save_path)  # 图片本地化
        thumb_img = ''.join(re.findall('<img.*src="(.*?\..*?)"', content)[:1])  # 缩略图
        distinct = distinct_url_verify(url)
        if title and (not distinct):
            res = save_data(
                title=title, superior=q_obj.class_lD, content=content, thumb_img=thumb_img, clicks=clicks,
                author=author, keyword=keyword, description=description, source=source)
            if res:
                distinct_detail_write(url=url, title=title, superior=q_obj.class_lD)
                return res  # 入库成功
        print('标题为空,程序自动过滤!')
